# -*- encoding: utf-8 -*-
# vim: set et sw=4 ts=4 sts=4 ff=unix fenc=utf8:

### 数据抓取脚本

import time
import re
from selenium.common.exceptions import StaleElementReferenceException
from selenium.common.exceptions import TimeoutException
import selenium

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC


from db.mysqlbase import *
sqldb = MysqlBase()


import six
from six import iteritems

def pretty_unicode(string):
    """
    Make sure string is unicode, try to decode with utf8, or unicode escaped string if failed.
    """
    if isinstance(string, six.text_type): return string
    try:
        return string.decode("utf8")
    except UnicodeDecodeError:
        return string.decode('Latin-1').encode('unicode_escape').decode("utf8")



######配置log
import logging
logger = logging.getLogger()
handler = logging.StreamHandler()
formatter = logging.Formatter('%(asctime)s %(name)-12s %(levelname)-8s %(message)s')
handler.setFormatter(formatter)
logger.addHandler(handler)
logger.setLevel(logging.INFO)


import datetime
today = datetime.date.today()
tomorrow = today + datetime.timedelta(days=1)


def parse_from_csv():
	##通过csv文件获取需要抓取酒店信息
	##########csv 文件格式信息################################
	## 酒店编号编号,酒店名称,美团酒店网址,抓取酒店网址 	
	## 例如:
	##  4984887,西安金莎国际酒店,url1,url2
	##########################################################

	### 返回文件格式
	## 
	###	[{'poiId':4984887,'des_url':''},] 

	# 抓取地址后面带着格式
	
	import csv
	_file_ = "hotel_info.csv"
	_hotel = []

	
	with open(_file_) as f:
		reader = csv.reader(f)
		## 去掉表头
		next(reader)
		for row in reader:
			_hotel_tmp = {}
			_hotel_tmp['poiId'] = row[0]
			_hotel_tmp['name'] = row[1]
			_hotel_tmp['des_url'] = row[3] + "/?pa=1&q-check-out=%s&tab=description&q-room-0-adults=2&YGF=3&q-check-in=%s&MGT=1&WOE=2&WOD=1&ZSX=1&SYE=3&q-room-0-children=0" % (str(tomorrow),str(today))
			_hotel.append(_hotel_tmp)
	return _hotel

def spider_hotel():
	## 爬取酒店信息
	logger.info('开始爬取酒店信息')
	from splinter import Browser

	from selenium import webdriver
	options = webdriver.ChromeOptions()
	## 删除chrome正受到自动测试软件的控制提示
	options.add_argument('disable-infobars')
	options.add_argument('user-agent=Mozilla/5.0 (iPhone; CPU iPhone OS 10_2_1 like Mac OS X) AppleWebKit/602.4.6 (KHTML, like Gecko) Version/10.0 Mobile/14D27 Safari/602.1')
	
	prefs = {"profile.managed_default_content_settings.images" : 2}
	options.add_experimental_option("prefs", prefs)


	##获取需要爬取酒店信息列表	
	_hotel_list_tmp = parse_from_csv()	
	_hotel_count = 0
	
	for i in range(0,len(_hotel_list_tmp),20):
		_hotel_list = _hotel_list_tmp[i: i+20]
	

		with Browser('chrome', options=options) as browser:	
			browser.driver.set_page_load_timeout(60)
			for _hotel in _hotel_list:
				_hotel_count += 1	
				logger.info(u'开始爬取第%d个酒店:' % _hotel_count + u' 酒店抓取地址为: ' + pretty_unicode(_hotel['des_url']))
				
				try:
					_driver = browser.visit(_hotel['des_url'])	
					#WebDriverWait(browser.driver, 10).until(EC.presence_of_element_located((By.CLASS_NAME, "room")))
				except TimeoutException:
					logger.info(u'访问页面超时,继续下一个页面访问')
					time.sleep(2)
					browser.driver.refresh();	
					continue
					#pass

				time.sleep(5)

#							from lib import utils
#							utils.send_mail_requests('825823776@qq.com', subject="网页刷新失败,请人工干预", text=None, html=None, async=False, _from="xiyoulaoyuanjia@gmail.com", attachment=None)


		
				## 解析元素,查找信息	
				for i in browser.driver.find_elements_by_css_selector(".room.cont.clearfix"):
					try:
						# 房型
						_fx = pretty_unicode(i.find_element_by_css_selector('.room-info h3').text)
						
						for j in i.find_elements_by_css_selector('.rateplan'):						
							# 房价
							_price = pretty_unicode(j.find_element_by_css_selector('.current-price').text)
							# 库存
							try:
								_kc = pretty_unicode(j.find_element_by_css_selector('.urgency-message').text)
							except selenium.common.exceptions.NoSuchElementException:
								_kc = u"无限"	
							# 早餐(无早餐／单份／双分)
							# 返回列表中从第二个开始都是不可见的, 所以text方法返回空 
							##
							try:
								_zc = pretty_unicode(j.find_element_by_css_selector('.breakfast .feature-title').text)
							except selenium.common.exceptions.NoSuchElementException:
								_zc = u'不含早餐'
					
							_str = u"抓取: 房型: %s, 有无早餐: %s, 房价为: %s, 库存为: %s" % (_fx, _zc, _price, _kc)
							logger.info(_str)

							_poiId = j.get_attribute("data-rateplan-id")
							###########更新数据库########
							### 现在把同程所有房型都抓回来
							## poiId,hotelname,fx,zc,kc, price, price_change
							_where = 'poiId=%s and fx="%s"' % (_poiId, _fx)
							try:
								 	item = sqldb.select(tablename=u"hotel",where=_where).next()
									logger.info("更新一条数据")
									_price_digte_old = re.findall('\d+', item['price'])[0]
									_price_digte_new = re.findall('\d+', _price)[0]
									_price_change = u"价格变化为: %.3f" % ((float(_price_digte_new) - float(_price_digte_old))/float(_price_digte_old))
									sqldb.update(tablename=u"hotel", where=_where, zc=_zc, kc=_kc, price=_price, price_change= _price_change, updatetime=time.time(), date=str(today))
							except StopIteration:
								logger.info("插入一条数据")
								sqldb.insert(tablename=u"hotel", poiId=_poiId, hotelname=pretty_unicode(_hotel['name'].decode('utf-8')),fx=_fx, zc=_zc, kc=_kc, price=_price, price_change=u"初始化数据", updatetime = time.time(), date=str(today))
					except StaleElementReferenceException:				
						browser.driver.refresh();	
				logger.info(u'爬取酒店:'+  u' 酒店抓取地址为: ' + pretty_unicode(_hotel['des_url']) + u' 结束' )


		logger.info('爬取酒店信息结束')

spider_hotel()

from apscheduler.schedulers.blocking import BlockingScheduler
from datetime import datetime

# BlockingScheduler
scheduler = BlockingScheduler()
#scheduler.add_job(spider_hotel, 'cron', day_of_week='1-5', hour=6, minute=30)
scheduler.add_job(spider_hotel,  'interval', hours=4 )
scheduler.start()
